package com.spark.core.transformation

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

/**
 * distinct 去重，有shuffle产生，内部实际是 map+reduceByKey+map实现
 */
object Demo15_distinct {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf()
    conf.setMaster("local")
    conf.setAppName("distinct")
    val sc = new SparkContext(conf)
    sc.setLogLevel("error")

    val infos = sc.parallelize(List[String]("a", "a", "b", "b", "c", "c", "d"), 4)
    // map+reduceByKey+map实现去重
//    val pairRdd: RDD[(String, Int)] = infos.map((_, 1))
//    val wcRdd: RDD[(String, Int)] = pairRdd.reduceByKey(_ + _)
//    val result: RDD[String] = wcRdd.map(_._1)

    val result: RDD[String] = infos.distinct()

    println("result: " + result.getNumPartitions)
    result.foreach(println)

  }
}
